{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n",
       " 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "C_neg_log_loss = -0.587763 #上一轮的得到的值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=5, missing=None, n_estimators=267,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=2, reg_lambda=0.0, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=0.3),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8], 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=267,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "xgb3_1.reg_alpha=2 #正则参数调优得到\n",
    "xgb3_1.reg_lambda=0.0 #正则参数调优得到\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58914, std: 0.00320, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58655, std: 0.00349, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58555, std: 0.00410, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58472, std: 0.00362, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58373, std: 0.00388, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58357, std: 0.00387, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58798, std: 0.00331, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58562, std: 0.00458, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58408, std: 0.00399, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58404, std: 0.00433, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58304, std: 0.00407, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58252, std: 0.00347, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58776, std: 0.00492, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58699, std: 0.00374, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58413, std: 0.00398, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58435, std: 0.00410, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58311, std: 0.00343, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58236, std: 0.00360, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58775, std: 0.00330, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58560, std: 0.00314, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58470, std: 0.00313, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58334, std: 0.00357, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58278, std: 0.00307, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58248, std: 0.00333, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       " -0.5823613868820933)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([109.27691832, 121.81335573, 128.76082087, 130.7491715 ,\n",
       "        124.8912559 , 116.70339098, 125.22021389, 140.27070532,\n",
       "        147.39503746, 149.70333829, 142.62184639, 135.29495268,\n",
       "        143.97569456, 159.8732182 , 170.65065055, 167.4646307 ,\n",
       "        155.35924797, 144.3362699 , 148.07247839, 166.01113605,\n",
       "        177.57208843, 175.68817859, 166.49167423, 141.07069182]),\n",
       " 'std_fit_time': array([2.92348469, 3.52073175, 2.7968601 , 3.37985197, 2.80997687,\n",
       "        3.06843076, 3.70800094, 3.18505871, 3.55651813, 4.73440909,\n",
       "        4.66641   , 3.61273944, 3.37406927, 3.96248238, 5.45489804,\n",
       "        4.59249709, 1.20604254, 2.0160922 , 2.73866388, 0.92608643,\n",
       "        0.19619494, 1.34412047, 1.64779365, 5.65424941]),\n",
       " 'mean_score_time': array([0.57291837, 0.49316969, 0.4812686 , 0.47728047, 0.51987891,\n",
       "        0.49283695, 0.52881398, 0.51847692, 0.47680211, 0.48039494,\n",
       "        0.51084828, 0.51034527, 0.49753723, 0.48098011, 0.4830596 ,\n",
       "        0.57245412, 0.4713273 , 0.47102671, 0.48044047, 0.47144041,\n",
       "        0.47052312, 0.46886978, 0.46924672, 0.33834305]),\n",
       " 'std_score_time': array([0.13063021, 0.02485621, 0.00669559, 0.00923235, 0.06510599,\n",
       "        0.02396351, 0.092692  , 0.0716671 , 0.00448715, 0.00511807,\n",
       "        0.06317672, 0.04848743, 0.01456462, 0.00833069, 0.00947876,\n",
       "        0.0860288 , 0.00357775, 0.00094413, 0.00294505, 0.00256847,\n",
       "        0.00317777, 0.00286785, 0.00779261, 0.00524573]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9,\n",
       "                    0.9, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " 'split0_test_score': array([-0.583143  , -0.58008444, -0.57860971, -0.57872177, -0.57702249,\n",
       "        -0.5774943 , -0.58279893, -0.57773468, -0.57716278, -0.57664097,\n",
       "        -0.5762907 , -0.57680442, -0.5789794 , -0.58049795, -0.57705783,\n",
       "        -0.57770355, -0.57746129, -0.57623822, -0.58212765, -0.5813842 ,\n",
       "        -0.57950542, -0.57746894, -0.5776187 , -0.57688531]),\n",
       " 'split1_test_score': array([-0.58892802, -0.58597666, -0.58348629, -0.58289381, -0.58225453,\n",
       "        -0.58093448, -0.58630097, -0.58425823, -0.58201285, -0.58290388,\n",
       "        -0.58116353, -0.58095756, -0.58637831, -0.58557017, -0.58267905,\n",
       "        -0.58255256, -0.58135227, -0.58033706, -0.58621477, -0.5829559 ,\n",
       "        -0.58329039, -0.58202864, -0.5815234 , -0.58082587]),\n",
       " 'split2_test_score': array([-0.59025058, -0.58766262, -0.58701109, -0.58546393, -0.58467699,\n",
       "        -0.58465089, -0.58986375, -0.58588808, -0.58650512, -0.58404538,\n",
       "        -0.58359715, -0.58275536, -0.58897201, -0.58797056, -0.58572026,\n",
       "        -0.58521313, -0.58362885, -0.58457422, -0.58876847, -0.58566229,\n",
       "        -0.58506188, -0.58388132, -0.58335373, -0.58379906]),\n",
       " 'split3_test_score': array([-0.59096016, -0.58928352, -0.58866115, -0.58765089, -0.58801131,\n",
       "        -0.58684172, -0.58833133, -0.58983813, -0.58711403, -0.58752904,\n",
       "        -0.58654385, -0.58627304, -0.59187834, -0.58992542, -0.58704083,\n",
       "        -0.58623242, -0.58635551, -0.58474416, -0.59022798, -0.58819931,\n",
       "        -0.58731416, -0.58499843, -0.58488189, -0.58447909]),\n",
       " 'split4_test_score': array([-0.59239677, -0.58974562, -0.58999217, -0.58887154, -0.58670385,\n",
       "        -0.58793519, -0.59259765, -0.59039212, -0.5876042 , -0.58908007,\n",
       "        -0.58763082, -0.58580611, -0.59261069, -0.5910116 , -0.58814858,\n",
       "        -0.59004086, -0.58673709, -0.58591435, -0.59141474, -0.58981939,\n",
       "        -0.58831136, -0.58830513, -0.58653677, -0.58642841]),\n",
       " 'mean_test_score': array([-0.58913551, -0.58655038, -0.58555181, -0.58472014, -0.58373365,\n",
       "        -0.58357105, -0.58797824, -0.58562196, -0.58407958, -0.58403956,\n",
       "        -0.58304493, -0.5825191 , -0.58776345, -0.5869949 , -0.58412906,\n",
       "        -0.58434816, -0.58310678, -0.58236139, -0.5877505 , -0.58560396,\n",
       "        -0.58469642, -0.58333619, -0.58278267, -0.58248331]),\n",
       " 'std_test_score': array([0.0031988 , 0.00349442, 0.00409811, 0.00362467, 0.00387917,\n",
       "        0.00386746, 0.00330618, 0.00457536, 0.00399071, 0.00432689,\n",
       "        0.00406782, 0.00346551, 0.00491868, 0.00373934, 0.00398175,\n",
       "        0.00410063, 0.00343382, 0.00359969, 0.00330378, 0.00314087,\n",
       "        0.00312974, 0.00357439, 0.00306845, 0.00332751]),\n",
       " 'rank_test_score': array([24, 19, 16, 15,  9,  8, 23, 18, 11, 10,  5,  3, 22, 20, 12, 13,  6,\n",
       "         1, 21, 17, 14,  7,  4,  2], dtype=int32),\n",
       " 'split0_train_score': array([-0.51744913, -0.51237423, -0.5114157 , -0.50861787, -0.50716007,\n",
       "        -0.50846498, -0.51439988, -0.51146311, -0.50697723, -0.5051586 ,\n",
       "        -0.50461006, -0.50508962, -0.51183894, -0.50774851, -0.50468848,\n",
       "        -0.50282819, -0.50135232, -0.50128676, -0.50936951, -0.50661729,\n",
       "        -0.50129087, -0.50123389, -0.50161325, -0.49928743]),\n",
       " 'split1_train_score': array([-0.51797157, -0.5126214 , -0.5101187 , -0.50733069, -0.50560292,\n",
       "        -0.50561734, -0.51428861, -0.50960189, -0.50668451, -0.50458788,\n",
       "        -0.50286306, -0.50261159, -0.51067975, -0.50719285, -0.50334406,\n",
       "        -0.50047911, -0.50055563, -0.50054573, -0.50821499, -0.50424456,\n",
       "        -0.50082194, -0.50034041, -0.4981713 , -0.49819304]),\n",
       " 'split2_train_score': array([-0.51736418, -0.51251153, -0.50853094, -0.50636358, -0.50620045,\n",
       "        -0.50559002, -0.51464236, -0.51025604, -0.50567352, -0.50334558,\n",
       "        -0.50315922, -0.50365118, -0.51077904, -0.50692231, -0.50404916,\n",
       "        -0.50190993, -0.49988444, -0.50036213, -0.50801249, -0.5032522 ,\n",
       "        -0.50040877, -0.4984408 , -0.49835619, -0.49800521]),\n",
       " 'split3_train_score': array([-0.5171269 , -0.51267947, -0.5086956 , -0.50763346, -0.50602786,\n",
       "        -0.50603588, -0.51389952, -0.50947496, -0.5061976 , -0.50367823,\n",
       "        -0.50336036, -0.50263452, -0.50938   , -0.50722018, -0.50235643,\n",
       "        -0.50098527, -0.50049047, -0.50128268, -0.50763921, -0.50359424,\n",
       "        -0.50088309, -0.49850607, -0.49809258, -0.49823064]),\n",
       " 'split4_train_score': array([-0.51510362, -0.51138886, -0.50807956, -0.50657992, -0.50554705,\n",
       "        -0.50451637, -0.51274916, -0.50919104, -0.50512523, -0.50414733,\n",
       "        -0.50218413, -0.50184803, -0.50854295, -0.5058484 , -0.50264947,\n",
       "        -0.50063366, -0.49950005, -0.49955379, -0.50775779, -0.50239949,\n",
       "        -0.50100798, -0.49964954, -0.49789297, -0.49791866]),\n",
       " 'mean_train_score': array([-0.51700308, -0.5123151 , -0.5093681 , -0.5073051 , -0.50610767,\n",
       "        -0.50604492, -0.5139959 , -0.50999741, -0.50613162, -0.50418352,\n",
       "        -0.50323537, -0.50316699, -0.51024414, -0.50698645, -0.50341752,\n",
       "        -0.50136723, -0.50035658, -0.50060622, -0.5081988 , -0.50402156,\n",
       "        -0.50088253, -0.49963414, -0.49882526, -0.498327  ]),\n",
       " 'std_train_score': array([0.000989  , 0.00047468, 0.0012304 , 0.0008055 , 0.00058165,\n",
       "        0.00131019, 0.0006679 , 0.00081176, 0.00067049, 0.0006439 ,\n",
       "        0.00079434, 0.00111916, 0.00115413, 0.00062885, 0.00086435,\n",
       "        0.00088337, 0.00063334, 0.00064677, 0.00061857, 0.00142781,\n",
       "        0.00028657, 0.00107283, 0.0014019 , 0.00049395])}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.582361 using {'colsample_bytree': 0.8, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.75, 0.8, 0.85], 'colsample_bytree': [0.75, 0.8, 0.85]}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [0.75,0.8,0.85]\n",
    "colsample_bytree = [0.75,0.8,0.85]\n",
    "param_test3_2 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=5, missing=None, n_estimators=267,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=2, reg_lambda=0.0, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=0.3),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'subsample': [0.75, 0.8, 0.85], 'colsample_bytree': [0.75, 0.8, 0.85]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 前面参数调整得到的最优值代入\n",
    "xgb3_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=267,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "xgb3_2.reg_alpha=2\n",
    "xgb3_2.reg_lambda=0.0\n",
    "gsearch3_2 = GridSearchCV(xgb3_2, param_grid = param_test3_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_2.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fei/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58240, std: 0.00309, params: {'colsample_bytree': 0.75, 'subsample': 0.75},\n",
       "  mean: -0.58247, std: 0.00372, params: {'colsample_bytree': 0.75, 'subsample': 0.8},\n",
       "  mean: -0.58226, std: 0.00389, params: {'colsample_bytree': 0.75, 'subsample': 0.85},\n",
       "  mean: -0.58322, std: 0.00397, params: {'colsample_bytree': 0.8, 'subsample': 0.75},\n",
       "  mean: -0.58236, std: 0.00360, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58268, std: 0.00331, params: {'colsample_bytree': 0.8, 'subsample': 0.85},\n",
       "  mean: -0.58283, std: 0.00347, params: {'colsample_bytree': 0.85, 'subsample': 0.75},\n",
       "  mean: -0.58223, std: 0.00396, params: {'colsample_bytree': 0.85, 'subsample': 0.8},\n",
       "  mean: -0.58203, std: 0.00379, params: {'colsample_bytree': 0.85, 'subsample': 0.85}],\n",
       " {'colsample_bytree': 0.85, 'subsample': 0.85},\n",
       " -0.5820257887681147)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_2.grid_scores_, gsearch3_2.best_params_,     gsearch3_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.85, 0.85)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取调优后的正则参数\n",
    "# neg_log_loss得分越大，模型越好\n",
    "\n",
    "#b步得到的neg_log_loss = -0.588041\n",
    "if gsearch3_2.best_score_> gsearch3_1.best_score_:\n",
    "    colsample_bytree_bst=gsearch3_2.best_params_['colsample_bytree']\n",
    "    subsample_bst=gsearch3_2.best_params_['subsample']\n",
    "else:\n",
    "    colsample_bytree_bst=gsearch3_1.best_params_['colsample_bytree']\n",
    "    subsample_bst=gsearch3_1.best_params_['subsample']\n",
    "\n",
    "if gsearch3_2.best_score_> C_neg_log_loss:\n",
    "    colsample_bytree_bst=gsearch3_2.best_params_['colsample_bytree']\n",
    "    subsample_bst=gsearch3_2.best_params_['subsample']\n",
    "else:\n",
    "    colsample_bytree_bst=0\n",
    "    subsample_bst=1\n",
    "    \n",
    "colsample_bytree_bst,subsample_bst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
